<?php

class Spider {

    public static function fromBaidu($strKw, $intPage = 1) {
        $pn = ($intPage - 1) * 10;
        $kwd = urlencode($strKw);
        $baiduUrl = 'http://www.baidu.com/s?wd=' . $kwd . '&pn=' . $pn . '&ie=utf-8&f=3';
		log::n('Fetching...:'.$baiduUrl);
        $snoopy = getSnoopy($baiduUrl);
        $snoopy->fetchlinks($baiduUrl);
        $result = snoopy_parse($snoopy, $baiduUrl, 'fetchlinks');
        if (!is_array($result)) {
            log::w('fetch links failed!');
            return false;
        }
        $arrReturn = array();

        //列表超链
        foreach ($result as $link) {
            if (substr($link, 0, strlen('http://www.baidu.com/link?url=')) == 'http://www.baidu.com/link?url=') {
                $arrReturn['tablelink'][] = $link;
            }
        }
        //分页超链
        foreach ($result as $link) {
            if (substr($link, 0, strlen('http://www.baidu.com/s?wd=' . $kwd)) == 'http://www.baidu.com/s?wd=' . $kwd && strpos($link, '&pn=')) {
                $arrReturn['pagelink'][] = $link;
            }
        }

        //相关关键字超链接
        $pRelationLink = '#http://www.baidu.com/s\?wd=(.*)&#isU';
        foreach ($result as $link) {
            preg_match_all($pRelationLink, $link, $d);
            if ($d[1][0] && $d[1][0] != $kwd && !strpos($link, 'tn=baidufir')) {
                $arrReturn['relationlink'][] = $link;
                $arrReturn['relationkw'][] = urldecode($d[1][0]);
            }
        }

        foreach ($arrReturn as &$arrLink) {
            $arrLink = array_unique($arrLink);
            $arrLink = array_filter($arrLink);
        }unset($arrLink);

        return $arrReturn;
    }

    public static function fromBaiduNews($strKw, $intPage = 1, $inTitle = false, $rn = 20) {
        $strKw = iconv('UTF-8', 'GB2312//IGNORE', $strKw);
        $pn = ($intPage - 1) * 10;
        $kwd = urlencode($strKw);
        if ($inTitle) {
            $kwd = 'title%3A(' . $kwd . ')';
        }
        $baiduUrl = 'http://news.baidu.com/ns?bt=0&et=0&si=&rn=' . $rn . '&tn=news&ie=gb2312&ct=1&word=' . $kwd . '&pn=' . $pn . '&cl=2';
        $snoopy = getSnoopy($baiduUrl, false);
        $snoopy->fetchlinks($baiduUrl);
        $result = snoopy_parse($snoopy, $baiduUrl, 'fetchlinks');

        if (!is_array($result)) {
            log::w('fetch links failed!');
            return false;
        }
        $arrReturn = array();

        foreach ($result as $link) {
            if (!strpos($link, 'baidu.com')) {
                $arrReturn['tablelink'][] = $link;
            }
            if (strpos($link, 'news.baidu.com') && strpos($link, '&pn=')) {
                $arrReturn['pagelink'][] = $link;
            }
            if (strpos($link, 'news.baidu.com') && strpos($link, '&tag=1')) {
                $arrReturn['relationlink'][] = $link;
            }
        }

        foreach ($arrReturn as &$arrLink) {
            $arrLink = array_unique($arrLink);
            $arrLink = array_filter($arrLink);
        }unset($arrLink);

        return $arrReturn;
    }

    public static function spiderFetchHtml($url) {
        return spiderFetchHtml($url);
    }

    public static function makeSQL($sqlfile, $fromWhere, $fromKeyword, $arrURLList = array()) {
        if (!file_exists($sqlfile)) {
            fCheckDir($sqlfile);
        }
        foreach ($arrURLList as $url) {
            $sArr = spiderFetchHtml($url);
            $data = array();
            if ($sArr['title'] && $sArr['body']) {
                $data['title'] = $sArr['title'];
                $data['body'] = $sArr['body'];
                $data['insertTime'] = date('Y-m-d H:i:s');
                global $dhttp_lastredirectaddr;
                $data['fromurl'] = $dhttp_lastredirectaddr;
                $data['fromWhere'] = $fromWhere;
                $data['fromKeyword'] = $fromKeyword;
                $query = makeInsertSQL('spider_knews_list', $data);
                file_put_contents($sqlfile, $query . PHP_EOL, 8);
            }
        }
    }

}

?>